import json
import os
import threading

from bs4 import BeautifulSoup
from xjlibrary.MRabbitMQ.useMQ.nodemq import Distributer, dealThread, Collecter
from xjlibrary.configread import MyConfigParser
from xjlibrary.mdatetime.mtime import getTodayDate
from xjlibrary.mrequest import baserequest
from xjlibrary.network.MyIP import GetLocalIPByPrefix
from xjlibrary.our_file_dir.base_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
config = BaseDir.get_new_path(BaseDir.get_upper_dir(curPath, -1), "db.ini")
TopPath = BaseDir.get_upper_dir(curPath, -3)
sPath = BaseDir.get_new_path(TopPath, "download", "sipo", "download", "pagejson")
# sPath = "E:\\down_data\\sipo\\pagejson"
GolobalConfig = MyConfigParser().set_config_path(config).get_configparser().set_keep_keys_case().read_config().conf_to_dict()
# print(GolobalConfig)
queueCenter2Node = GolobalConfig['get_article']['MQQueueCenter2Node']
MQQueueNode2Center = GolobalConfig['get_article']['MQQueueNode2Center']

HEARDER = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
    "Cache-Control": "no-cache",
    "Content-Type": "application/x-www-form-urlencoded",
    "Host": "epub.sipo.gov.cn",
    "Origin": "http://epub.sipo.gov.cn",
    "Pragma": "no-cache",
    "Proxy-Connection": "keep-alive",
    "Referer": "http://epub.sipo.gov.cn/gjcx.jsp",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36"
}

url = "http://epub.sipo.gov.cn/patentoutline.action"

postdata = {
    "showType": "1",
    "strSources": "",
    "strWhere": "OPD=BETWEEN['1985.09.01','1985.10.01']",  # 日期
    "numSortMethod": "",
    "strLicenseCode": "",
    "numIp": "0",
    "numIpc": "",
    "numIg": "0",
    "numIgc": "",
    "numIgd": "",
    "numUg": "0",
    "numUgc": "",
    "numUgd": "",
    "numDg": "0",
    "numDgc": "",
    "pageSize": "10",  # 每页显示量 3或者10
    "pageNow": "1"  # 此处为page
}


def down_page_func(result_queue, proxylist, starttime, endtime, page):
    postdata["strWhere"] = "OPD=BETWEEN['{}','{}']".format(starttime, endtime)
    postdata["pageNow"] = str(page)
    BoolResult = None
    while True:
        try:
            proxy = proxylist.pop()
            if not proxy:
                continue
        except:
            break
        proxys = {
            "http": proxy,
            "https": proxy
        }
        print(proxys)
        # print(postdata)
        BoolResult, errString, r = baserequest.BaseRequestPost(url, data=postdata, headers=HEARDER, mark="w790 right",
                                                               proxies=proxys,
                                                               timeout=(30, 60))
        if BoolResult:
            print("请求成功")
            content = r.content
            if content.startswith(b'\xef\xbb\xbf'):  # 去掉 utf8 bom 头
                content = content[3:]
            jsonText = content.decode('utf8').strip()
            # print('jsonText: %s' % jsonText)
            # dic = json.loads(jsonText)

            doc = BeautifulSoup(jsonText, "lxml")
            boxDivTags = doc.find_all("div", class_="cp_box")
            boolWrite = False
            for box in boxDivTags:
                liTags = box.find_all("li", class_="wl228")
                for li in liTags:
                    if li.get_text().find("申请公布号：") != -1:
                        cnnum = li.get_text().strip().replace("申请公布号：", "").strip()
                    if li.get_text().find("申请公布日：") != -1:
                        date = li.get_text().strip().replace("申请公布日：", "").strip()
                        if starttime <= date <= endtime:
                            sql = "insert ignore into `index`(`cnnum`,`date`) value ('{}','{}')".format(cnnum,date)
                            print("**************************")
                            print(sql)
                            result_queue.put(sql)
                            boolWrite = False
                        else:
                            boolWrite = True
                            break
                if boolWrite:
                    break
            if boolWrite:
                sql = "update eapopage set stat=0 where starttime='{}' and endtime='{}' and `page`='{}'".format(
                    starttime,
                    endtime,
                    page)
                result_queue.put(sql)
                break

            sumDict = dict()
            sumDict["strWhere"] = "OPD=BETWEEN['{}','{}']".format(starttime, endtime)
            sumDict["pageNow"] = str(page)
            sumDict['html'] = jsonText
            # 每个线程单独写入一个文件，无需加锁
            if not os.path.exists(sPath):
                os.makedirs(sPath)
            outPathFile = os.path.join(sPath,
                                       '%s_%s_%d_%d.big_json' % (
                                           getTodayDate(), GetLocalIPByPrefix('192.168.'), os.getpid(),
                                           threading.get_ident())
                                       )
            print('Write to %s ...' % outPathFile)
            with open(outPathFile, mode='a', encoding='utf-8') as f:
                line = json.dumps(sumDict, ensure_ascii=False).strip() + '\n'
                f.write(line)
            sql = "update eapopage set stat=1 where starttime='{}' and endtime='{}' and `page`='{}'".format(starttime,
                                                                                                            endtime,
                                                                                                            page)
            result_queue.put(sql)
            break
        else:
            print("请求失败")
    if not BoolResult:
        sql = "update eapopage set stat=0,failcount = failcount + 1 where starttime='{}' and endtime='{}' and `page`='{}'".format(
            starttime,
            endtime, page)
        result_queue.put(sql)


class downPage(dealThread):

    def __init__(self, num, func):
        super(downPage, self).__init__(num, func)

    def parameters(self, result):
        dic = json.loads(result)
        proxy = dic['proxy']
        starttime = dic['starttime']
        endtime = dic['endtime']
        page = dic['page']
        return [proxy], {'starttime': starttime, 'endtime': endtime, 'page': page}


if __name__ == "__main__":
    threadNum = GolobalConfig['get_article']['WorkerThreadNumber']
    Distributer(config, queueCenter2Node, 5).start()
    Collecter(config, MQQueueNode2Center).start()
    down = downPage(int(threadNum), down_page_func)
    down.setfunc(down_page_func)
    down.run()
